# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv
from utils.HttpUtils import HttpUtils


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告


"""
CNN
https://edition.cnn.com/
"""


class NewYorkTimes():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # "User-Agent": ua.random
            # 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15',
            'authority': 'samizdat-graphql.nytimes.com',
            'method': 'POST',
            'path': '/graphql/v2',
            'scheme': 'https',
            'accept': '*/*',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'content-type': 'application/json',
            'cookie': 'nyt-gdpr=0; nyt-a=5yhbzzKEPiPkMiH33YbkS8; nyt-purr=cfhhcfh; nyt-geo=HK; __gads=ID=d6f41aaf1bb50914:T=1587207203:S=ALNI_MbtVMa-PSFk_Hec0d_kFhRzUl4rig; purr-cache=<K0<r<C_<G_<S0; edu_cig_opt=%7B%22isEduUser%22:false%7D; b2b_cig_opt=%7B%22isCorpUser%22:false%7D; _gcl_au=1.1.894336198.1587207204; walley=GA1.2.477823072.1587207203; walley_gid=GA1.2.1793182238.1587207207; iter_id=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhaWQiOiI1ZTlhZGMyZmVkMzAzODAwMDExYmEwZGIiLCJjb21wYW55X2lkIjoiNWMwOThiM2QxNjU0YzEwMDAxMmM2OGY5IiwiaWF0IjoxNTg3MjA3MjE1fQ.58tV8uJPrKbryEEndxC5aMztvMZDiywe5ltBikakTXg; nyt-us=0; datadome=P6CIsf94EJ5fCaoWSOD~qFfeLarRby~jDa9JhOSaaMMV4ess5NG5dR27fTJlPtQSE9ORd6L~2DK1qDlFX3pWyWIS2MYdrQXOpYh5xsGFWu; nyt-m=A6EE314F728F73E1AB4427ABE69FE3B3&n=i.2&vr=l.4.0.0.0.0&cav=i.1&l=l.1.2122340221&g=i.0&er=i.1587207680&iub=i.0&ifv=i.0&imv=i.1&iru=i.0&v=i.1&pr=l.4.0.0.0.0&fv=i.0&imu=i.1&iue=i.0&iir=i.0&t=i.1&vp=i.0&ft=i.0&igu=i.1&igf=i.0&uuid=s.ffc09441-7594-437b-b781-d7d7bd406b50&ica=i.0&rc=i.0&ier=i.0&igd=i.0&iga=i.0&ird=i.0&ira=i.0&s=s.core&e=i.1588291200&prt=i.0; nyt-jkidd=uid=0&lastRequest=1587207680372&activeDays=%5B0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C1%5D&adv=1&a7dv=1&a14dv=1&a21dv=1&lastKnownType=anon; _gat_UA-58630905-2=1',
            'dnt': '1',
            'nyt-app-type': 'project-vi',
            'nyt-app-version': '0.0.5',
            'nyt-token': 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs+/oUCTBmD/cLdmcecrnBMHiU/pxQCn2DDyaPKUOXxi4p0uUSZQzsuq1pJ1m5z1i0YGPd1U1OeGHAChWtqoxC7bFMCXcwnE1oyui9G1uobgpm1GdhtwkR7ta7akVTcsF8zxiXx7DNXIPd2nIJFH83rmkZueKrC4JVaNzjvD+Z03piLn5bHWU6+w+rA+kyJtGgZNTXKyPh6EC6o5N+rknNMG5+CdTq35p8f99WjFawSvYgP9V64kgckbTbtdJ6YhVP58TnuYgr12urtwnIqWP9KSJ1e5vmgf3tunMqWNm6+AnsqNj8mCLdCuc5cEB74CwUeQcP2HQQmbCddBy2y0mEwIDAQAB',
            'origin': 'https://www.nytimes.com',
            'referer': 'https://www.nytimes.com/search?query=COVID',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-site',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36'

        }


    def get_contents(self, cursor):
        postData = {
            'size':'10',
            'q':'COVID',  # 关键词
            'from':'10',
            'page':cursor,  # 页号
        }
        url = f"https://samizdat-graphql.nytimes.com/graphql/v2"
        str_data = f'{{"operationName":"SearchRootQuery","variables":{{"first":10,"sort":"best","text":"COVID-19","filterQuery":"","sectionFacetFilterQuery":"","typeFacetFilterQuery":"","sectionFacetActive":false,"typeFacetActive":false,"cursor":"YXJyYXljb25uZWN0aW9uOjk="}},"extensions":{{"persistedQuery":{{"version":1,"sha256Hash":"d632cb222f6a7dd48349d5789975e28fbf7017963fe1ec16fd20fa868b335842"}}}}}}'
        html = HttpUtils.do_request("POST", url, self.headers, str_data)
        data_json = json.loads(html.text)
        count = data_json['data']['search']['hits']['totalCount']  # 总页数
        next_cursor = data_json['data']['search']['hits']['pageInfo']['endCursor']  # 下页游标
        for d in data_json['data']['search']['hits']['edges']:
            dict_data = dict()
            dict_data['url'] = d['node']['node']['url']  # 标题
            dict_data['title'] = d['node']['node']['promotionalHeadline']  # 标题
            dict_data['desc'] = d['node']['node']['promotionalSummary']  # 简介
            dict_data['desc'] = d['node']['node']['firstPublished']  # 发布时间

            print(str(dict_data))


if __name__ == '__main__':
    newyorktimes = NewYorkTimes()
    for i in range(1, 100):
        print(f"============{i}=============")
        newyorktimes.get_contents(i)